package it.uniroma3.dia.ia.textcategorization.datasource.odpdump;

import it.uniroma3.dia.ia.textcategorization.datasource.DataSource;
import it.uniroma3.dia.ia.textcategorization.model.Category;
import it.uniroma3.dia.ia.textcategorization.model.Text;
import it.uniroma3.dia.ia.textcategorization.utils.MarkupRemover;
import it.uniroma3.dia.ia.textcategorization.utils.PageRetriever;

import java.util.HashSet;
import java.util.Iterator;
import java.util.Set;

public final class ODPDump implements DataSource {
	
	private Set<String> cachedLinks = null;
	private Category categoryForCachedLinks = null;

	@Override
	public Set<Category> getTopLevelCategories() {
		return ODPDumpTopLevelCategories.getCategories();
	}

	@Override
	public Set<String> getSubcategoriesNamesFor(Category category) {
		Set<String> subcategoriesNames = null;
		String path = PathUtility.composePath(category);
		subcategoriesNames = SubcategoriesFinder.getSubcategoriesNamesFor(path);
		return subcategoriesNames;
	}

	@Override
	public long getNumberOfTextsAvailableFor(Category category) {
		String path = PathUtility.composePath(category);
		Set<String> links = LinksCollector.getLinks(path);
		this.cachedLinks = links;
		this.categoryForCachedLinks = category;
		return countLinks(links);
	}

	@Override
	public Set<Text> getTexts(Category category, long numberOfTexts) {
		Set<Text> texts = new HashSet<Text>();
		
		Set<String> links = tryToGetLinksFromCache(category);
		
		long retrievedTexts = 0;
		Iterator<String> iterator = links.iterator();
		while (retrievedTexts < numberOfTexts && iterator.hasNext()) {
			String link = iterator.next();
			Text text = retrieve(link);
			if (text != null) {
				texts.add(text);
				retrievedTexts++;
			}
		}
		
		return texts;
	}

	private Text retrieve(String link) {
		Text result = null;
		
		Text page = PageRetriever.retrieve(link);
		if (page != null) {
			Text text = MarkupRemover.removeMarkup(page);
			if (text != null) {
				result = text;
			}
		}
			
		return result;
	}

	private Set<String> tryToGetLinksFromCache(Category category) {
		Set<String> links = null;
		if (category.equals(this.categoryForCachedLinks)) {
			links = this.cachedLinks;
		} else {
			String path = PathUtility.composePath(category);
			links = LinksCollector.getLinks(path);
		}
		return links;
	}

	private long countLinks(Set<String> links) {
		long count = 0;
		Iterator<String> iterator = links.iterator();
		while (iterator.hasNext()) {
			count++;
			iterator.next();
		}
		return count;
	}

}
